from jili.core.printlog import print
import datetime
import os
import multiprocessing
from jili.tool.convert import str2datetime
from jili.core import load,save
from jili.data.db import get_calender
import pandas as pd
from jili.data.db import getdb_client
from jili.tool.state import config
from multiprocessing import Manager
class dataset:
    """
    data综合工具：
    1.数据读取迭代
    2.数据更新保存
    3.各种类型数据都能处理
    4.各种品种数据都能处理
    5.离散小工具：数据最大日期，
    """
    def __init__(self,product="future",datatype="k1d",baseurl=r"F:",region="C",fqtype="hfq",source=None,runmode="pkl"):
        """
        Args:
            product: stock,future,fund,index,bond,coin
            datatype: k1d,k1m,tick
            baseurl: price的根目录   r"F:/price"
            region: C,A,H   非C的 会自动拼接成  stock_A
            fqtype: hfq,bfq  跟回测中的数据对应，如果不存在合适的数据则需要计算复权数据
            source:数据源，专用拼接
            runmode：pkl文件，shm:共享内存，未来支持：redis,等内存数据库
        """
        self.baseurl=baseurl
        self.region=region
        self.product=product
        self.datatype=datatype
        self.fqtype=fqtype
        self.data_tmp={}#数据缓存
        self.factors={}
        self.db=None
        self.source=source
        if datatype in ["k1m","tick"]:
            self.freq = "%Y%m%d"
        else:
            self.freq = "%Y%m"
        self.url=baseurl
        if region=="C":
            self.url=os.path.join(self.url,self.product)
        else:
            self.url = os.path.join(self.url, self.product+"_"+self.region)
        self.url = os.path.join(self.url, self.datatype)
        if source:
            self.url = self.url+"_"+source
        if self.product in ["stock","fund"]:
            self.url = os.path.join(self.url, self.fqtype)
        self.isavailable=False
        if os.path.exists(self.url):
            l=os.listdir(self.url)
            if len(l)>0:
                self.isavailable=True
        self.runmode=runmode
        self.k1dtool=None
    def get_k1d_bydate(self,obj,d):
        if self.k1dtool is None:
            self.k1dtool=dataset(product=self.product,datatype="k1d",baseurl=self.baseurl,region=self.region,fqtype=self.fqtype,source=self.source)
        data=self.k1dtool.get_data_bydate(obj,d)
        return data
    def get_objs(self):
        objs=os.listdir(self.url)
        return objs
    #基础工具  获取最新日期，最新timekey,最新行情
    def get_maxdate(self,obj):#obj  单个合约或是  all
        maxdate=None
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                if self.freq=="%Y%m%d":
                    maxdate = l.split(".")[0].split("_")[-1]
                else:
                    d = load(os.path.join(url0, l))
                    maxdate = max(list(d.keys()))
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    maxdate = l.split(".")[0].split("_")[-1]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    maxdate = max(list(d.keys()))
        return maxdate
    def get_startdate(self,obj):#obj  单个合约或是  all
        maxdate=None
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = min(os.listdir(url0))
                if self.freq=="%Y%m%d":
                    maxdate = l.split(".")[0].split("_")[-1]
                else:
                    d = load(os.path.join(url0, l))
                    maxdate = min(list(d.keys()))
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = min(os.listdir(url0))
                    maxdate = l.split(".")[0].split("_")[-1]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = min(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    maxdate = min(list(d.keys()))
        return maxdate
    def get_datelist(self,obj):
        datelist = []
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
        else:
            url0 = os.path.join(url, obj)
        if os.path.exists(url0):
            l = os.listdir(url0)
            l.sort()
            if self.freq == "%Y%m%d":
                for i in l:
                    d = i.split(".")[0].split("_")[-1]
                    datelist.append(d)
            else:
                d0 = load(os.path.join(url0, l[0]))
                d0 = min(list(d0.keys()))
                d1 = load(os.path.join(url0, l[-1]))
                d1 = min(list(d1.keys()))
                for d in get_calender(d0, d1):
                    datelist.append(d)
        return datelist

    def get_maxtimekey(self,obj):
        maxtimekey = None
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                d = load(os.path.join(url0, l))
                maxdate = max(list(d.keys()))
                maxtimekey=d[maxdate][-1]["timekey"]
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    maxtimekey = d[-1]["timekey"]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    maxdate = max(list(d.keys()))
                    if self.datatype == "k1d":
                        maxtimekey = d[maxdate]["timekey"]
                    else:
                        dd = d[maxdate]
                        maxtimekey = dd[-1]["timekey"]
        return maxtimekey
    def get_lastdata(self,obj):
        data = None
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                d = load(os.path.join(url0, l))
                maxdate = max(list(d.keys()))
                data=d[maxdate][-1]
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    data = d[-1]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = load(os.path.join(url0, l))
                    maxdate = max(list(d.keys()))
                    if self.datatype == "k1d":
                        data = d[maxdate]
                    else:
                        dd = d[maxdate]
                        data = dd[-1]
        return data
    #获取数，按日期，按开始结束日期需要增加复权数据计算
    def get_factor(self,obj,timekey):
        timekey=pd.to_datetime(timekey)
        if obj  in self.factors.keys():
            if timekey in self.factors[obj].keys():
                return self.factors[obj][timekey]
        if self.db is None:
            self.db=getdb_client(dbname="stock",ip=config.mongodb_ip)
        t=1
        for i in self.db["stock_right_factor"].find({"obj":obj,"timekey":{"$lte":timekey}}).sort([("timekey",-1)]):
            if obj not in self.factors.keys():
                self.factors[obj]={}
            # t=i["adjustFactor"]
            t=i
            self.factors[obj][timekey]=t
            break
        return t
    def get_data_bydate(self,obj,d):#具有快速缓存
        data = []
        d0=str2datetime(d)
        url = self.url
        if isinstance(obj,str):
            if obj in self.data_tmp.keys():
                if d0 in self.data_tmp[obj].keys():
                    return self.data_tmp[obj][d0]
            if obj == "all":
                url0 = os.path.join(url, "_all", "all_" + d0.strftime(self.freq) + ".pkl")
            else:
                url0 = os.path.join(url, obj, obj + "_" + d0.strftime(self.freq) + ".pkl")
            if os.path.exists(url0):
                if self.freq == "%Y%m%d":
                    data = load(url0)
                else:
                    data0=load(url0)
                    if obj!="all":
                        self.data_tmp[obj]=data0
                    if d0 in data0.keys():
                        data=data0[d0]
        else:
            if d0 in self.data_tmp.keys():
                data0=self.data_tmp[d]
                for obj0 in obj:
                    if obj0 in data0.keys():
                        v=data0[obj0]
                        if isinstance(v,list):
                            data=data+v
                        else:
                            data.append(v)
                    else:#load
                        url0 = os.path.join(url, obj0, obj0 + "_" + d0.strftime(self.freq) + ".pkl")
                        if os.path.exists(url0):
                            if self.freq == "%Y%m%d":
                                v = load(url0)
                                data = data + v
                            else:
                                data0 = load(url0)
                                for d1,v in data0.items():
                                    if d1 not in self.data_tmp.keys():
                                        self.data_tmp[d1]={}
                                    self.data_tmp[d1][obj0] = v[0]
                                    if d1==d0:
                                        data.append(v[0])
            else:#load
                for obj0 in obj:
                    url0 = os.path.join(url, obj0, obj0 + "_" + d0.strftime(self.freq) + ".pkl")
                    if os.path.exists(url0):
                        if self.freq == "%Y%m%d":
                            v = load(url0)
                            data = data + v
                        else:
                            data0 = load(url0)
                            for d1, v in data0.items():
                                if d1 not in self.data_tmp.keys():
                                    self.data_tmp[d1] = {}
                                self.data_tmp[d1][obj0] = v[0]
                                if d1 == d0:
                                    data.append(v[0])
        return data
    def get_data(self,obj,start=None,end=None):
        """
        Args:
            obj:
            start:date
            end:date
        Returns:
            按日的行情集  如果是个股就是list,如果是all，就是按timekey的dict
        """
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
        else:
            url0 = os.path.join(url, obj)
        data = []
        if os.path.exists(url0):
            l=os.listdir(url0)
            l.sort()
            if start:
                d0=str2datetime(start)
                start0=obj+"_"+d0.strftime(self.freq)+".pkl"
            else:
                start0=None
            if end:
                d1=str2datetime(end)
                end0=obj+"_"+d1.strftime(self.freq)+".pkl"
            else:
                end0=None
            if start0:
                if end0:
                    for i in l:
                        if i>=start0 and i<=end0:
                            url1=os.path.join(url0,i)
                            dset=load(url1)
                            if i==start0:
                                for d,v in dset.items():
                                    if d>=d0:
                                        data=data+v
                            elif i==end0:
                                for d,v in dset.items():
                                    if d<=d1:
                                        data=data+v
                            else:
                                # data.update(dset)
                                for d,v in dset.items():
                                    data=data+v
                else:
                    for i in l:
                        if i<=start0:
                            url1=os.path.join(url0,i)
                            dset=load(url1)
                            if i==start0:
                                for d,v in dset.items():
                                    if d>=d0:
                                        data=data+v
                            else:
                                # data.update(dset)
                                for d, v in dset.items():
                                    data=data+v
            else:
                if end0:
                    for i in l:
                        if i<=end0:
                            url1=os.path.join(url0,i)
                            dset=load(url1)
                            if i==end0:
                                for d,v in dset.items():
                                    if d<=d1:
                                        data=data+v
                            else:
                                # data.update(dset)
                                for d, v in dset.items():
                                    data=data+v
                else:
                    for i in l:
                        url1=os.path.join(url0,i)
                        dset=load(url1)
                        # data.update(dset)
                        for d, v in dset.items():
                            data=data+v
        else:
            url0=os.path.join(url, obj)
            if os.path.exists(url0):
                l=os.listdir(url0)
                l.sort()
                if start:
                    start=str2datetime(start)
                    if end:
                        end=str2datetime(end)
                        isstart = False
                        startstr = start.strftime(self.freq)
                        isend=False
                        for k in l:
                            if startstr in k:
                                isstart = True
                            if isstart:
                                bars=load(os.path.join(url, obj, k))
                                for d,i in bars.items():
                                    if d >= start:
                                        if d<=end:
                                            data = data + i
                                        else:
                                            isend=True
                                            break
                                if isend:
                                    break
                    else:
                        isstart=False
                        startstr=start.strftime(self.freq)
                        for k in l:
                            if startstr in k:
                                isstart=True
                            if isstart:
                                bars = load(os.path.join(url, obj, k))
                                for d, i in bars.items():
                                    if d >= start:
                                        if d <= end:
                                            data = data + i
                else:
                    for k in l:
                        bars = load(os.path.join(url, obj, k))
                        for d, i in bars.items():
                            data = data + i
        return data
    #保存数据 普通模式与 ——all模式
    def save_byday(self,obj,data,url=None):
        if url   is None:
            url = self.url
        data0={}
        for kk,v in data.items():
            k=kk.strftime(self.freq)
            url0 = os.path.join(url, obj, obj + "_" + k + ".pkl")
            if self.freq == "%Y%m%d":
                if os.path.exists(url0):
                    data = load(url0)
                    keys = []
                    for i in v:
                        keys.append(i["timekey"])
                    for i in reversed(data):
                        if i["timekey"] not in keys:
                            v.insert(0, i)
                save(v,url0)
        #     else:
        #
        #         if os.path.exists(url0):
        #             data = load(url0)
        #             data.update(v)
        #             v = data
        # if data0:
        #     save(v, url0)
    def save(self,obj,data,url=None):
        if url   is None:
            url = self.url
        d={}
        if isinstance(data,list):
            for i in data:
                tk=i["timekey"]
                key=tk.strftime(self.freq)
                if self.freq=="%Y%m%d":
                    if key not in d.keys():
                        d[key]=[]
                    d[key].append(i)
                else:
                    if key not in d.keys():
                        d[key]={}
                    date0=datetime.datetime(tk.year,tk.month,tk.day)
                    if date0 not in d[key].keys():
                        d[key][date0] = []
                    d[key][date0].append(i)
        elif isinstance(data,dict):
            for k,i in data.items():
                if isinstance(k,datetime.datetime):
                    tk=k
                else:
                    tk = i["timekey"]
                key = tk.strftime(self.freq)
                if self.freq == "%Y%m%d":
                    if key not in d.keys():
                        d[key] = []
                    d[key].append(i)
                else:
                    if key not in d.keys():
                        d[key]={}
                    date0=datetime.datetime(tk.year,tk.month,tk.day)
                    if date0 not in d[key].keys():
                        d[key][date0] = []
                    d[key][date0].append(i)
        for k,v in d.items():
            url0=os.path.join(url,obj,obj+"_"+k+".pkl")
            if self.freq == "%Y%m%d":
                if os.path.exists(url0):
                    data=load(url0)
                    keys=[]
                    for i in v:
                        keys.append(i["timekey"])
                    for i in reversed(data):
                        if i["timekey"] not in keys:
                            v.insert(0,i)
            else:
                if os.path.exists(url0):
                    data=load(url0)
                    data.update(v)
                    v=data
            save(v,url0)
    def save_byall(self,datas):
        if not datas:
            return None
        l=list(datas.keys())
        if isinstance(l[0],datetime.datetime):
            prek=None
            data={}
            l.sort()
            for d in l:
                v=datas[d]
                k = d.strftime(self.freq)
                if prek!=k:
                    if data:
                        url0 = os.path.join(self.url, "_all", "all_" + k + ".pkl")
                        if os.path.exists(url0):
                            data0 = load(url0)
                            data0.update(data)
                            # for d0,vv in data.items():
                            #     if d0 in data0.keys():
                            #         data0[d0].update(vv)
                            #     else:
                            #         data0[d0] = vv
                            save(data0, url0)
                        else:
                            save(data,url0)
                        data={}
                else:
                    data[d]=v
                prek=k
            if data:
                url0 = os.path.join(self.url, "_all", "all_" + k + ".pkl")
                if os.path.exists(url0):
                    data0 = load(url0)
                    data0.update(data)
                    # for d0, vv in data.items():
                    #     if d0 in data0.keys():
                    #         data0[d0].update(vv)
                    #     else:
                    #         data0[d0] = vv
                    save(data0, url0)
                else:
                    save(data, url0)
        else:
            for k , v in datas.items():
                url0=os.path.join(self.url,"_all","all_"+k+".pkl")
                if os.path.exists(url0):
                    data = load(url0)
                    data.update(v)
                    save(data,url0)
                else:
                    save(v,url0)
    #迭代器，主要是回测用，需要增加复权数据计算
    def get_pkl_byday(self,url0,obj,start,end):
        for d in get_calender(start, end, self.region):
            url1 = os.path.join(url0, obj + "_" + d.strftime(self.freq) + ".pkl")
            datset = load(url1)
            if datset:
                for i in datset:
                    yield self.datatype, i
            else:
                print("sys", "不存在", obj, d, url1)
    def iter_price(self,obj,start=None,end=None):
        """
        Args:
            obj: 支持obj,objs:有 _all目录的会更快些
            start: 开始日期，
            end: 结束日期

        Returns:正常一笔行情；考虑是否需要按因子一批行情给出一批，然后各自处理？
        """
        url = self.url
        if isinstance(obj,str):
            if obj != "all":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    if self.freq == "%Y%m%d":
                        for d in get_calender(start,end,self.region):
                            url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                            datset = load(url1)
                            if datset:
                                for i in datset:
                                    yield self.datatype,i
                            else:
                                print("sys","不存在",obj,d,url1)
                    else:
                        l = os.listdir(url0)
                        l.sort()
                        if start:
                            start = str2datetime(start)
                            if end:
                                end = str2datetime(end)
                                isstart = False
                                startstr = start.strftime(self.freq)
                                isend = False
                                for k in l:
                                    if startstr in k:
                                        isstart = True
                                    if isstart:
                                        url0=os.path.join(url, obj, k)
                                        datset=load(url0)
                                        for k,bars in datset.items():
                                            for i in bars:
                                                if i["timekey"] >= start:
                                                    if i["timekey"] <= end:
                                                        yield self.datatype,i
                                                    else:
                                                        isend = True
                                                        break
                                        if isend:
                                            break
                            else:
                                isstart = False
                                startstr = start.strftime(self.freq)
                                for k in l:
                                    if startstr in k:
                                        isstart = True
                                    if isstart:
                                        url0 = os.path.join(url, obj, k)
                                        datset = load(url0)
                                        for k, bars in datset.items():
                                            for i in bars:
                                                if i["timekey"] >= start:
                                                    yield self.datatype,i
                        else:
                            for k in l:
                                url0 = os.path.join(url, obj, k)
                                datset = load(url0)
                                for k, bars in datset.items():
                                    for i in bars:
                                        yield self.datatype,i
                else:
                    print("sys","不存在",url0)
            else:
                url0 = os.path.join(url, "_all")
                if os.path.exists(url0):
                    l = os.listdir(url0)
                    l.sort()
                    if start:
                        d0 = str2datetime(start)
                        start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                    else:
                        start0 = None
                    if end:
                        d1 = str2datetime(end)
                        end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                    else:
                        end0 = None
                    if start0:
                        if end0:
                            for i in l:
                                if start0 <= i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    elif i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                if i >= start0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                    else:
                        if end0:
                            for i in l:
                                if i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                url1 = os.path.join(url0, i)
                                dset = load(url1)
                                for d, v in dset.items():
                                    for i in v:
                                        yield self.datatype,i
                else:  # 临时拼接
                    objs = os.listdir(url)
                    for d in get_calender(start=start, end=end, region=self.region):
                        data=[]
                        for obj in objs:
                            data0=self.get_data_bydate(obj,d)
                            if data0:
                                data=data+data0
                    if data:
                        data=pd.DataFrame(data)
                        data.sort_values(by="timekey",inplace=True)
                        data=data.to_dict()
                        for i in data.values():
                            if i["obj"] in obj:
                                yield self.datatype,i
        else:
            url0 = os.path.join(url, "_all")
            isall=len(obj)>=100
            if isall and os.path.exists(url0):
                l = os.listdir(url0)
                l.sort()
                if start:
                    d0 = str2datetime(start)
                    start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                else:
                    start0 = None
                if end:
                    d1 = str2datetime(end)
                    end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                else:
                    end0 = None
                if start0:
                    if end0:
                        for i in l:
                            if i >= start0 and i <= end0:
                                url1 = os.path.join(url0, i)
                                dset = load(url1)
                                if i == start0:
                                    for d, v in dset.items():
                                        if d >= d0:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                elif i == end0:
                                    for d, v in dset.items():
                                        if d <= d1:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:
                        for i in l:
                            if i >= start0:
                                url1 = os.path.join(url0, i)
                                dset = load(url1)
                                if i == start0:
                                    for d, v in dset.items():
                                        if d >= d0:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                else:
                    if end0:
                        for i in l:
                            if i <= end0:
                                url1 = os.path.join(url0, i)
                                dset = load(url1)
                                if i == end0:
                                    for d, v in dset.items():
                                        if d <= d1:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:
                        for i in l:
                            url1 = os.path.join(url0, i)
                            dset = load(url1)
                            for d, v in dset.items():
                                for i in v:
                                    if i["obj"] in obj:
                                        yield self.datatype,i
            else:  # 临时拼接
                for d in get_calender(start=start, end=end, region=self.region):
                    data = []
                    for obj0 in obj:
                        data0 = self.get_data_bydate(obj0, d)
                        if data0:
                            data.extend(data0)
                    if data:
                        data = pd.DataFrame(data)
                        data.sort_values(by="timekey", inplace=True)
                        data = data.to_dict("index")
                        for i in data.values():
                            yield self.datatype,i
    def iter_data_addk1d(self,obj,start=None,end=None):#策略框架使用，k1d必须，结算需要，日前日后触发需要
        url = self.url
        if self.datatype=="k1d":
            for i in self.iter_price(obj,start,end):
                yield "k1d",i
        else:
            k1d_dataset=dataset(product=self.product,datatype="k1d",baseurl=self.baseurl,region=self.region,fqtype=self.fqtype,source=self.source)
            if isinstance(obj,str):
                if obj != "all":
                    url0 = os.path.join(url, obj)
                    if os.path.exists(url0):
                        if self.freq == "%Y%m%d":
                            for d in get_calender(start, end, self.region):
                                url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                                datset = load(url1)
                                if datset:
                                    for i in datset:
                                        yield self.datatype, i
                                else:
                                    print("sys", "不存在", obj, d, url1)
                                k1d0=k1d_dataset.get_data_bydate(obj,d)
                                yield "k1d",k1d0
                        else:
                            preurl1=None
                            for d in get_calender(start, end, self.region):
                                url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                                datset = {}
                                if url1!=preurl1:
                                    datset=load(url1)
                                    preurl1=url1
                                if d in datset.keys():
                                    for i in datset[d]:
                                        yield self.datatype, i
                                else:
                                    print("sys", "不存在", obj, d, url1)
                                k1d0=k1d_dataset.get_data_bydate(obj,d)
                                yield "k1d",k1d0
                    else:
                        print("sys","不存在",obj,url0)
                else:
                    url0 = os.path.join(url, "_all")
                    if os.path.exists(url0):
                        l = os.listdir(url0)
                        l.sort()
                        if start:
                            d0 = str2datetime(start)
                            start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                        else:
                            start0 = None
                        if end:
                            d1 = str2datetime(end)
                            end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                        else:
                            end0 = None
                        if start0:
                            if end0:
                                for i in l:
                                    if i <= start0 and i <= end0:
                                        url1 = os.path.join(url0, i)
                                        dset = load(url1)
                                        if i == start0:
                                            for d, v in dset.items():
                                                if d >= d0:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        elif i == end0:
                                            for d, v in dset.items():
                                                if d <= d1:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                            else:
                                for i in l:
                                    if i <= start0:
                                        url1 = os.path.join(url0, i)
                                        dset = load(url1)
                                        if i == start0:
                                            for d, v in dset.items():
                                                if d >= d0:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                        else:
                            if end0:
                                for i in l:
                                    if i <= end0:
                                        url1 = os.path.join(url0, i)
                                        dset = load(url1)
                                        if i == end0:
                                            for d, v in dset.items():
                                                if d <= d1:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                            else:
                                for i in l:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:  # 临时拼接
                        objs = os.listdir(url)
                        for d in get_calender(start=start, end=end, region=self.region):
                            data=[]
                            for obj in objs:
                                data0=self.get_data_bydate(obj,d)
                                if data0:
                                    data=data.extend(data0)
                        if data:
                            data=pd.DataFrame(data)
                            data.sort_values(by="timekey",inplace=True)
                            data=data.to_dict()
                            for i in data.values():
                                if i["obj"] in obj:
                                    yield self.datatype,i
            else:
                url0 = os.path.join(url, "_all")
                if os.path.exists(url0):
                    l = os.listdir(url0)
                    l.sort()
                    if start:
                        d0 = str2datetime(start)
                        start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                    else:
                        start0 = None
                    if end:
                        d1 = str2datetime(end)
                        end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                    else:
                        end0 = None
                    if start0:
                        if end0:
                            for i in l:
                                if i <= start0 and i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    elif i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                if i <= start0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                    else:
                        if end0:
                            for i in l:
                                if i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = load(url1)
                                    if i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                url1 = os.path.join(url0, i)
                                dset = load(url1)
                                for d, v in dset.items():
                                    for i in v:
                                        yield self.datatype,i
                else:  # 临时拼接
                    objs = os.listdir(url)
                    for d in get_calender(start=start, end=end, region=self.region):
                        data = []
                        for obj in objs:
                            data0 = self.get_data_bydate(obj, d)
                            if data0:
                                data = data.extend(data0)
                    if data:
                        data = pd.DataFrame(data)
                        data.sort_values(by="timekey", inplace=True)
                        data = data.to_dict()
                        for i in data.values():
                            yield self.datatype,i
    #历史数据导入导出工具
    def export_data1(self,obj,freq1,url0,url1):
        try:
            print("deal",obj)
            l = os.listdir(os.path.join(url0, obj))
            l.sort()
            prek = None
            data = {}
            for i in l:
                d = i.split(".")[0].split("_")[-1]
                d = str2datetime(d)
                key = d.strftime(freq1)
                if prek   is None:
                    prek = key
                url = os.path.join(url0, obj, i)
                data0 = load(url)
                if prek == key:
                    data[d] = data0
                else:
                    url11 = os.path.join(url1, obj, obj + "_" + prek + ".pkl")
                    save(data, url11)
                    prek = key
                    data = {}
                    data[d] = data0
            url11 = os.path.join(url1, obj, obj + "_" + prek + ".pkl")
            save(data, url11)
        except Exception as e:
            print(url)
            print(e)
    def export_data2(self,obj,url0,url1):
        print("deal",obj)
        try:
            l = os.listdir(os.path.join(url0, obj))
            l.sort()
            for i in l:
                url = os.path.join(url0, obj, i)
                data = load(url)
                self.save(obj, data, url=url1)
        except Exception as e:
            print(url)
            print(e)
    def export_new_data(self,freq0,freq1,url0,url1,obj0=None,pool_n=1):
        objs=os.listdir(url0)
        isrun=True
        if obj0:
            isrun=False
        if pool_n>1:
            pool = multiprocessing.Pool(pool_n)
        if len(freq0)>len(freq1):
            for obj in objs:
                if not isrun:
                    if obj0==obj:
                        isrun=True
                    else:
                        print("跳过",obj)
                        continue
                if pool_n > 1:
                    pool.apply_async(self.export_data1,(obj,freq1,url0,url1))
                else:
                    self.export_data1(obj,freq1,url0,url1)

        elif len(freq0)<len(freq1):
            for obj in objs:
                if not isrun:
                    if obj0==obj:
                        isrun=True
                    else:
                        print("跳过",obj)
                        continue
                if pool_n > 1:
                    pool.apply_async(self.export_data2,(obj, url0, url1))
                else:
                    self.export_data2(obj, url0, url1)

        if pool_n > 1:
            pool.close()  # 关闭进程池，不再接受新的进程
            pool.join()  # 主进程阻塞等待子进程的退出

